import requests
import parsel
import os
import concurrent.futures


def get_resp(html_url):
    """发送请求"""
    global headers
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
    }
    resp = requests.get(url=html_url, headers=headers)
    # 返回相应对象
    return resp


def get_link_url(link):
    """获取图集详情页的url"""

    # 先调用发送请求的函数
    html_data = get_resp(link).text
    # 转换数据类型
    selector_1 = parsel.Selector(html_data)
    # 提取url地址
    href_list = selector_1.css('.contlistw .cl li a::attr(href)').getall()
    return href_list


def get_img_info(html_url):
    """获取图片数据 url 标题"""
    # 调用发送请求的函数
    resp = get_resp(html_url)
    # 转换数据类型解析数据
    selector = parsel.Selector(resp.text)
    # 提取图片标题
    title = selector.css('.showcontw #showimg img::attr(alt)').get()
    # 提取图片url
    img_url_list = selector.css('.piclistw li a img::attr(src)').getall()
    return title, img_url_list


def save(title, img_url_list):
    """保存数据"""
    # 创建文件路径
    file = 'img2\\'
    # 判断这个文件是否存在
    if not os.path.exists(file):
        # 自动创建文件夹
        os.mkdir(file)

    num = 1

    for img_url in img_url_list:
        img_url = img_url.replace('thumb_200_0_', '')
        name = img_url.split('/')[-1]

        # 保存数据
        resp_1 = requests.get(url=img_url, headers=headers).content
        # with open(f'img/{title}/{name}', mode='wb')as f:
        with open(file + title + str(num) + '.jpg', mode='wb') as f:
            f.write(resp_1)
        # print(f'{img_url}正在下载')
        num += 1

        print(title, img_url)

def main(page):
    """主函数"""

    # 获取图集详情页的url地址
    link = f'https://www.3gbizhi.com/meinv/mnxz_{page}.html'

    # 调用函数
    href = get_link_url(link)
    for index in href:
        title, img_url_list = get_img_info(index)
        # 保存数据
        save(title, img_url_list)


if __name__ == '__main__':
    # 多线程 线程池
    exe = concurrent.futures.ThreadPoolExecutor(max_workers=10)
    for page in range(1, 17):
        exe.submit(main, page)
        # main(page)
    # 关闭
    exe.shutdown()







